Samples

library(modelr)
library(recipes)

Basic samples

Perform a basic 70:30 split

set.seed(20180409)
brent_dt %>% 
  modelr::resample_partition(c(train=0.7,test=0.3)) ->
  splits
splits %>% 
  pluck("train") %>% 
  as_tibble()->
  train_raw
splits %>% 
  pluck("test") %>% 
  as_tibble()->
  test_raw

Start our recipe for processing our data

train_raw %>% 
  recipe( .)  %>%
  add_role(any_purchase, new_role = "outcome") %>% 
  add_role(email_user_id, new_role = "id") %>% 
  add_role(everything(),-any_purchase, -email_user_id, new_role = "predictor")->
  starter_recipe
starter_recipe
Data Recipe

Inputs:

      role #variables
        id          1
   outcome          1
 predictor        188

Perform some steps to remove columns and fill in missing country

starter_recipe %>% 
  step_rm(has_role("id")) %>% 
  step_corr(all_numeric())  %>% 
  step_bagimpute(country_code, seed_val = 42) %>% 
  step_other(country_code) %>%  
  step_zv(all_predictors())  %>% 
  prep(train_raw) ->
  filter_recipe
filter_recipe
Data Recipe

Inputs:

      role #variables
        id          1
   outcome          1
 predictor        188

Training data contained 69362 data points and 13773 incomplete rows. 

Operations:

Variables removed email_user_id [trained]
Correlation filter removed h10_clicks, h11_clicks, h6_clicks, h8_clicks, ... [trained]
Bagged tree imputation for country_code [trained]
Collapsing factor levels for country_code [trained]
Zero variance filter removed no terms [trained]
train_b <- bake(filter_recipe, newdata = train_raw) 
test_b  <- bake(filter_recipe, newdata = test_raw) 
train_b

Perform numeric standardisation steps.

train_b %>% 
  recipe(any_purchase~.) %>% 
  step_center(all_numeric()) %>% 
  step_scale(all_numeric()) %>% 
  prep(train_b) ->
  standardise_recipe
standardise_recipe
Data Recipe

Inputs:

      role #variables
   outcome          1
 predictor        135

Training data contained 69362 data points and no missing data.

Operations:

Centering for member_rating, holiday_open_prop, ... [trained]
Scaling for member_rating, holiday_open_prop, ... [trained]
train_std <- bake(standardise_recipe, train_b) 
test_std  <- bake(standardise_recipe, test_b) 
train_std

Oversample training purchases

set.seed(20180101)
filter_recipe %>% 
  step_upsample(all_outcomes(), ratio= .25) %>% 
  prep(retain=TRUE) %>% 
  juice() %>% 
  # hack because juice isn't reducing the column set
  bake(filter_recipe, .) ->
  train_up
print(paste("New rows:", nrow(train_up)- nrow(train_b)))
[1] "New rows: 15726"

Now standardise it

train_ups <- bake(standardise_recipe, train_up) 

Synthesised training purchases

library(synthpop)
brent_dt %>% 
  filter(any_purchase=="TRUE") %>% 
  bake(filter_recipe,.) %>% 
  syn(k=30000)  ->
  synth_purchases
Sample(s) of size 30000 will be generated from original data of size 1836.

Variable any_purchase has only one value so its method has been changed to "constant".
Variable any_purchase removed as predictor because only one value.
syn  variables
1    member_rating country_code any_purchase X2016.04.Early.Access.Coupon X6.Month.DBA.Training.Plan Blog.Posts DBAreactions.Tuesdays...Fridays First.Responder.Kit.Updates Monday.Links...our.favorite.SQL...tech.news.from.the.week SQL.Server.Updates
     Unknown.recd X2017.05.Intersection.Orlando.Pre.Con X2017.05.SQLDay.Poland.Pre.Con X6.Month.DBA.Plan Clients Contact.Form Dell.DBA.Days.2016 EBook.Setup FeedBurner First.Responder.Kit
     iTunes.Rating MailOrderDBAs.com Members OptinMonster...BrentOzar.com Sitewide.Footer SQLServerUpdates.com Unknown.Signup.Source..Before.2014.Sept. Unknown.source Watch.Brent.Tune.Queries Webcast.Drawing
     Webcast.Registration holiday_open_prop weekday_open_raw weekday_open_prop weekend_open_raw weekend_open_prop h0_opens h1_opens h10_opens h11_opens
     h12_opens h13_opens h14_opens h2_opens h3_opens h4_opens h5_opens h6_opens h7_opens h8_opens
     h9_opens h0_opensactive h1_opensactive h10_opensactive h11_opensactive h12_opensactive h13_opensactive h14_opensactive h2_opensactive h3_opensactive
     h4_opensactive h5_opensactive h6_opensactive h7_opensactive h8_opensactive h9_opensactive h0_opensprop h1_opensprop h10_opensprop h11_opensprop
     h12_opensprop h13_opensprop h14_opensprop h2_opensprop h3_opensprop h4_opensprop h5_opensprop h6_opensprop h7_opensprop h8_opensprop
     h9_opensprop holiday_clicks_prop weekday_clicks_prop weekend_clicks_raw weekend_clicks_prop h0_clicks h1_clicks h12_clicks h13_clicks h14_clicks
     h2_clicks h3_clicks h4_clicks h7_clicks h0_clickactive h1_clickactive h10_clickactive h11_clickactive h12_clickactive h13_clickactive
     h14_clickactive h2_clickactive h3_clickactive h4_clickactive h5_clickactive h6_clickactive h7_clickactive h8_clickactive h9_clickactive h0_clickprop
     h1_clickprop h10_clickprop h11_clickprop h12_clickprop h13_clickprop h14_clickprop h2_clickprop h3_clickprop h4_clickprop h5_clickprop
     h6_clickprop h7_clickprop h8_clickprop h9_clickprop topic_8_B_clicks topic_NA_B_clicks topic_NA_nB_clicks topic_8_B_prop topic_8_nB_prop topic_NA_B_prop
     topic_NA_nB_prop topic_8_B_active topic_NA_B_active topic_NA_nB_active never_opened never_clicked
synth_purchases %>% 
  pluck("syn") %>% 
  union(train_b) %>% 
  sample_n(nrow(.)) ->
  train_syn
train_syn_std<- bake(standardise_recipe,train_syn) 

Preparing

We’ll want to build models against our training sets so let’s save some code by making a big list we can map models to.

train_sets<-list(
  "Basic"=train_b,
  "Standardised"=train_std,
  "Upsampled"=train_up,
  "Upsampled & Standardised"=train_ups,
  "Synthesised"=train_syn,
  "Synthesised & Standardised" =train_syn_std
)

Let’s remove a high RAM object.

rm("filter_recipe")

Models - Basic glms

library(broom)
library(ggplot2)
train_sets %>% 
  map(~glm(any_purchase~  member_rating+ First.Responder.Kit+ Unknown.recd+ Monday.Links...our.favorite.SQL...tech.news.from.the.week+ weekday_open_prop + weekday_clicks_prop, 
           data=.,
           family="binomial",
           y=FALSE,x=FALSE,model=FALSE))  ->
  basic_glm

saveRDS(basic_glm, "../outputs/glms.rds")
basic_glm %>% 
  map_df(tidy, .id = "set") %>% 
  filter(term!="(Intercept)") %>% 
  ggplot(aes(x=term, y=estimate, colour=set)) +
  geom_point(alpha=.5, size=3)  +
  coord_flip() +
  ggthemes::theme_fivethirtyeight()+
  geom_hline(aes(yintercept=0), colour="darkgrey", linetype="dashed")

Attributes with values consistently above 0 (dashed line) increase likelihood to buy training, and those below reduce the likelihood. The further away from the 0, the greater the impact.

library(optiRum)
vals_to_conv<-seq(-1.5,1,by=0.5)
data_frame(Coefficient=vals_to_conv,
           `Odds Ratio (p/p-1)`= round(logit.odd(vals_to_conv),2),
           `Probability` = round(logit.prob(vals_to_conv),2))

Models - Basic tree models

library(FFTrees)
train_sets %>% 
  map(~dplyr::select(mutate(., any_purchase=as.logical(any_purchase)),
              member_rating:weekend_open_prop, 
              holiday_clicks_prop:weekend_clicks_prop)) %>% 
  map(~FFTrees(any_purchase~., .,
               goal="bacc",do.comp = FALSE,progress = FALSE)) ->
  basic_trees
saveRDS(basic_trees, "../outputs/trees.rds")
basic_trees %>% 
  map_df(~cbind(.$tree.definitions,.$tree.stats$train),.id = "set") ->
  basic_tree_results
basic_tree_results

The chart shows different trees built using the different data sets and the balanced accuracy measure. Balanced accuracy is average of the proportion of purchases correctly classified and the proportion of non-purchases correctly classified.

basic_tree_results %>% 
  ggplot(aes(x=cues, y=bacc, colour=set)) +
  geom_point(alpha=.5, size=3)  +
  coord_flip() +
  ggthemes::theme_fivethirtyeight()

We can look at the difference between the correctly classified purchases (red) and the correctly classified non-purchases (grey) for the trees with the highest balanced accuracy. There’s two visible cases:

  1. high accuracy for purchases and low accuracy for non-purchases
  2. OK accuracy for purchases and good accuracy for non-purchases

Selection one of these would depend on the cost of the low accuracy for non-purchases.

basic_tree_results %>% 
  top_n(10, bacc) %>% 
  mutate(cues=stringr::str_wrap(stringr::str_replace_all(cues,";", " "),1)) %>% 
  dplyr::select(cues, sens, spec, set) %>% 
  ggplot(aes(x=set, y=spec, ymax=sens, ymin=spec)) +
  geom_pointrange(alpha=0.3, size=2)+
  geom_point(aes(y=sens), colour="red", size=6)+
  facet_wrap(~cues, ncol=4)+
  ggthemes::theme_fivethirtyeight()

Models - glmnets

glmnets take all our variables and construct a model with them. However, a model with 135 coefficients would be pretty crazy. Instead glmnet downweights coefficients towards 0 as it penalises complexity. As such, even though our model notionally contains all our columns not everything will contribute.

library(glmnet)
library(glmnetUtils)
train_sets %>% 
  map(~cv.glmnet(any_purchase~., data=. ,
              family = "binomial",type.measure="class",
              standardize = FALSE)) ->
  basic_glmnets

saveRDS(basic_glmnets, "../outputs/glmnets.rds")
basic_glmnets %>% 
  map_df(~rownames_to_column(as.data.frame(as.matrix(coefficients(.))),"col"),.id = "set" ) %>% 
  rename(coef=`1`) -> 
  basic_glmnet_coefs
basic_glmnet_coefs %>% 
  filter(coef!=0.000000e+00) %>% 
  filter(col!="(Intercept)") %>% 
  arrange(col) %>% 
  count(col) %>% 
  mutate( pane = row_number() %/% 38) ->
  panes
  
basic_glmnet_coefs %>% 
  inner_join(panes) %>% 
  arrange(desc(col)) %>% 
  mutate(col=fct_inorder(col)) %>% 
  ggplot(aes(x=col, y=coef, colour=set)) +
  geom_point() + 
  coord_flip()+
  facet_wrap(~pane,scales = "free", ncol=5)+
  ggthemes::theme_fivethirtyeight()

The non-standardised types of samples do not work well with glmnet and typically it would have scaled them. We can see which columns (primarily in the standardised samples) make it our models most commonly. These will be the most predictive columns overall.

panes %>% 
  arrange(desc(n)) %>% 
  mutate(col=fct_inorder(col)) %>% 
  mutate( pane = row_number() %/% 38) %>% 
  arrange(desc(col)) %>% 
  mutate(col=fct_inorder(col)) %>% 
  ggplot(aes(x=col, y=n)) +
  geom_col() + 
  coord_flip()+
  facet_wrap(~pane, ncol=5, scales = "free_y")+
  ggthemes::theme_fivethirtyeight()

Models - Complex models via h2o

library(h2o)
options("h2o.use.data.table"=TRUE)
h2o.init(max_mem_size = "8G")

H2O is not running yet, starting it now...

Note:  In case of errors look at the following log files:
    C:\Users\steph\AppData\Local\Temp\RtmpCynpf4/h2o_steph_started_from_r.out
    C:\Users\steph\AppData\Local\Temp\RtmpCynpf4/h2o_steph_started_from_r.err

java version "1.8.0_161"
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)

Starting H2O JVM and connecting: . Connection successful!

R is connected to the H2O cluster: 
    H2O cluster uptime:         2 seconds 157 milliseconds 
    H2O cluster version:        3.16.0.2 
    H2O cluster version age:    4 months and 12 days !!! 
    H2O cluster name:           H2O_started_from_R_steph_xwr366 
    H2O cluster total nodes:    1 
    H2O cluster total memory:   7.11 GB 
    H2O cluster total cores:    8 
    H2O cluster allowed cores:  8 
    H2O cluster healthy:        TRUE 
    H2O Connection ip:          localhost 
    H2O Connection port:        54321 
    H2O Connection proxy:       NA 
    H2O Internal Security:      FALSE 
    H2O API Extensions:         Algos, AutoML, Core V3, Core V4 
    R Version:                  R version 3.4.4 (2018-03-15) 
train_sets %>% 
  map(as.h2o) %>% 
  map(~h2o.automl(y = "any_purchase", training_frame = ., seed = 1313, max_runtime_secs = 180)) ->
  basic_automl

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |===                                                                                      |   3%
  |                                                                                               
  |====                                                                                     |   5%
  |                                                                                               
  |=====                                                                                    |   5%
  |                                                                                               
  |=====                                                                                    |   6%
  |                                                                                               
  |======                                                                                   |   6%
  |                                                                                               
  |======                                                                                   |   7%
  |                                                                                               
  |=======                                                                                  |   7%
  |                                                                                               
  |=======                                                                                  |   8%
  |                                                                                               
  |========                                                                                 |   8%
  |                                                                                               
  |========                                                                                 |   9%
  |                                                                                               
  |========                                                                                 |  10%
  |                                                                                               
  |=========                                                                                |  10%
  |                                                                                               
  |==========                                                                               |  11%
  |                                                                                               
  |==========                                                                               |  12%
  |                                                                                               
  |===========                                                                              |  12%
  |                                                                                               
  |===========                                                                              |  13%
  |                                                                                               
  |============                                                                             |  13%
  |                                                                                               
  |============                                                                             |  14%
  |                                                                                               
  |=============                                                                            |  14%
  |                                                                                               
  |=============                                                                            |  15%
  |                                                                                               
  |==============                                                                           |  15%
  |                                                                                               
  |==============                                                                           |  16%
  |                                                                                               
  |===============                                                                          |  16%
  |                                                                                               
  |===============                                                                          |  17%
  |                                                                                               
  |================                                                                         |  18%
  |                                                                                               
  |=================                                                                        |  19%
  |                                                                                               
  |=================                                                                        |  20%
  |                                                                                               
  |==================                                                                       |  20%
  |                                                                                               
  |==================                                                                       |  21%
  |                                                                                               
  |===================                                                                      |  21%
  |                                                                                               
  |===================                                                                      |  22%
  |                                                                                               
  |====================                                                                     |  22%
  |                                                                                               
  |====================                                                                     |  23%
  |                                                                                               
  |=====================                                                                    |  23%
  |                                                                                               
  |=====================                                                                    |  24%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |===                                                                                      |   3%
  |                                                                                               
  |=====                                                                                    |   5%
  |                                                                                               
  |=====                                                                                    |   6%
  |                                                                                               
  |======                                                                                   |   6%
  |                                                                                               
  |======                                                                                   |   7%
  |                                                                                               
  |=======                                                                                  |   7%
  |                                                                                               
  |=======                                                                                  |   8%
  |                                                                                               
  |========                                                                                 |   9%
  |                                                                                               
  |=========                                                                                |  10%
  |                                                                                               
  |=========                                                                                |  11%
  |                                                                                               
  |==========                                                                               |  11%
  |                                                                                               
  |==========                                                                               |  12%
  |                                                                                               
  |===========                                                                              |  12%
  |                                                                                               
  |===========                                                                              |  13%
  |                                                                                               
  |============                                                                             |  13%
  |                                                                                               
  |============                                                                             |  14%
  |                                                                                               
  |=============                                                                            |  14%
  |                                                                                               
  |=============                                                                            |  15%
  |                                                                                               
  |==============                                                                           |  15%
  |                                                                                               
  |==============                                                                           |  16%
  |                                                                                               
  |===============                                                                          |  16%
  |                                                                                               
  |===============                                                                          |  17%
  |                                                                                               
  |================                                                                         |  18%
  |                                                                                               
  |=================                                                                        |  19%
  |                                                                                               
  |=================                                                                        |  20%
  |                                                                                               
  |==================                                                                       |  20%
  |                                                                                               
  |==================                                                                       |  21%
  |                                                                                               
  |===================                                                                      |  21%
  |                                                                                               
  |===================                                                                      |  22%
  |                                                                                               
  |====================                                                                     |  22%
  |                                                                                               
  |====================                                                                     |  23%
  |                                                                                               
  |=====================                                                                    |  23%
  |                                                                                               
  |=====================                                                                    |  24%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |===                                                                                      |   3%
  |                                                                                               
  |====                                                                                     |   5%
  |                                                                                               
  |=====                                                                                    |   5%
  |                                                                                               
  |=====                                                                                    |   6%
  |                                                                                               
  |======                                                                                   |   7%
  |                                                                                               
  |=======                                                                                  |   8%
  |                                                                                               
  |========                                                                                 |   8%
  |                                                                                               
  |========                                                                                 |   9%
  |                                                                                               
  |=========                                                                                |  10%
  |                                                                                               
  |==========                                                                               |  11%
  |                                                                                               
  |==========                                                                               |  12%
  |                                                                                               
  |===========                                                                              |  12%
  |                                                                                               
  |===========                                                                              |  13%
  |                                                                                               
  |============                                                                             |  13%
  |                                                                                               
  |============                                                                             |  14%
  |                                                                                               
  |=============                                                                            |  14%
  |                                                                                               
  |=============                                                                            |  15%
  |                                                                                               
  |==============                                                                           |  15%
  |                                                                                               
  |==============                                                                           |  16%
  |                                                                                               
  |===============                                                                          |  16%
  |                                                                                               
  |===============                                                                          |  17%
  |                                                                                               
  |================                                                                         |  18%
  |                                                                                               
  |=================                                                                        |  19%
  |                                                                                               
  |=================                                                                        |  20%
  |                                                                                               
  |==================                                                                       |  20%
  |                                                                                               
  |==================                                                                       |  21%
  |                                                                                               
  |===================                                                                      |  21%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |===                                                                                      |   3%
  |                                                                                               
  |====                                                                                     |   5%
  |                                                                                               
  |=====                                                                                    |   5%
  |                                                                                               
  |=====                                                                                    |   6%
  |                                                                                               
  |======                                                                                   |   7%
  |                                                                                               
  |=======                                                                                  |   8%
  |                                                                                               
  |========                                                                                 |   8%
  |                                                                                               
  |========                                                                                 |   9%
  |                                                                                               
  |=========                                                                                |  10%
  |                                                                                               
  |==========                                                                               |  11%
  |                                                                                               
  |==========                                                                               |  12%
  |                                                                                               
  |===========                                                                              |  12%
  |                                                                                               
  |===========                                                                              |  13%
  |                                                                                               
  |============                                                                             |  13%
  |                                                                                               
  |============                                                                             |  14%
  |                                                                                               
  |=============                                                                            |  14%
  |                                                                                               
  |=============                                                                            |  15%
  |                                                                                               
  |==============                                                                           |  15%
  |                                                                                               
  |==============                                                                           |  16%
  |                                                                                               
  |===============                                                                          |  16%
  |                                                                                               
  |===============                                                                          |  17%
  |                                                                                               
  |================                                                                         |  18%
  |                                                                                               
  |=================                                                                        |  19%
  |                                                                                               
  |=================                                                                        |  20%
  |                                                                                               
  |==================                                                                       |  20%
  |                                                                                               
  |==================                                                                       |  21%
  |                                                                                               
  |===================                                                                      |  21%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |===                                                                                      |   3%
  |                                                                                               
  |====                                                                                     |   5%
  |                                                                                               
  |=====                                                                                    |   5%
  |                                                                                               
  |=====                                                                                    |   6%
  |                                                                                               
  |======                                                                                   |   6%
  |                                                                                               
  |======                                                                                   |   7%
  |                                                                                               
  |=======                                                                                  |   7%
  |                                                                                               
  |=======                                                                                  |   8%
  |                                                                                               
  |========                                                                                 |   8%
  |                                                                                               
  |========                                                                                 |   9%
  |                                                                                               
  |========                                                                                 |  10%
  |                                                                                               
  |=========                                                                                |  10%
  |                                                                                               
  |=========                                                                                |  11%
  |                                                                                               
  |==========                                                                               |  11%
  |                                                                                               
  |==========                                                                               |  12%
  |                                                                                               
  |===========                                                                              |  12%
  |                                                                                               
  |===========                                                                              |  13%
  |                                                                                               
  |============                                                                             |  13%
  |                                                                                               
  |============                                                                             |  14%
  |                                                                                               
  |=============                                                                            |  14%
  |                                                                                               
  |=============                                                                            |  15%
  |                                                                                               
  |==============                                                                           |  15%
  |                                                                                               
  |==============                                                                           |  16%
  |                                                                                               
  |===============                                                                          |  16%
  |                                                                                               
  |===============                                                                          |  17%
  |                                                                                               
  |================                                                                         |  18%
  |                                                                                               
  |=================                                                                        |  19%
  |                                                                                               
  |=================                                                                        |  20%
  |                                                                                               
  |==================                                                                       |  20%
  |                                                                                               
  |=====================================================================================    |  95%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |===                                                                                      |   3%
  |                                                                                               
  |=====                                                                                    |   5%
  |                                                                                               
  |=====                                                                                    |   6%
  |                                                                                               
  |======                                                                                   |   6%
  |                                                                                               
  |======                                                                                   |   7%
  |                                                                                               
  |=======                                                                                  |   7%
  |                                                                                               
  |=======                                                                                  |   8%
  |                                                                                               
  |========                                                                                 |   8%
  |                                                                                               
  |========                                                                                 |   9%
  |                                                                                               
  |========                                                                                 |  10%
  |                                                                                               
  |=========                                                                                |  10%
  |                                                                                               
  |=========                                                                                |  11%
  |                                                                                               
  |==========                                                                               |  11%
  |                                                                                               
  |==========                                                                               |  12%
  |                                                                                               
  |===========                                                                              |  12%
  |                                                                                               
  |===========                                                                              |  13%
  |                                                                                               
  |============                                                                             |  13%
  |                                                                                               
  |============                                                                             |  14%
  |                                                                                               
  |=============                                                                            |  14%
  |                                                                                               
  |=============                                                                            |  15%
  |                                                                                               
  |==============                                                                           |  15%
  |                                                                                               
  |==============                                                                           |  16%
  |                                                                                               
  |===============                                                                          |  16%
  |                                                                                               
  |===============                                                                          |  17%
  |                                                                                               
  |================                                                                         |  18%
  |                                                                                               
  |=================                                                                        |  19%
  |                                                                                               
  |=================                                                                        |  20%
  |                                                                                               
  |==================                                                                       |  20%
  |                                                                                               
  |=====================================================================================    |  95%
  |                                                                                               
  |=========================================================================================| 100%

  |                                                                                               
  |                                                                                         |   0%
  |                                                                                               
  |=========================================================================================| 100%
saveRDS(basic_automl, "../outputs/automl.rds")

Evaluation

LS0tDQp0aXRsZTogIkdldHRpbmcgYWxsIHVwIGluIEJyZW50J3MgYml6IC0gQ2FuZGlkYXRlIE1vZGVscyINCm91dHB1dDogDQogIGh0bWxfbm90ZWJvb2s6IA0KICAgIGNvZGVfZm9sZGluZzogaGlkZQ0KICAgIHRvYzogeWVzDQogICAgdG9jX2RlcHRoOiAxDQotLS0NCg0KYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9DQprbml0cjo6b3B0c19jaHVuayRzZXQod2FybmluZyA9IEZBTFNFLCBtZXNzYWdlID0gRkFMU0UpDQpsaWJyYXJ5KHRpZHl2ZXJzZSkNCmxpYnJhcnkoZHRwbHlyKQ0KYnJlbnRfZHQ8LWRhdGEudGFibGU6OmZyZWFkKCIuLi9vdXRwdXRzL2ZpbmFsZGF0YXNldC5jc3YiKQ0KYnJlbnRfZHRbLCBwdXJjaGFzZTo9TlVMTF0NCmJyZW50X2R0ICU+JSANCiAgYXNfdGliYmxlKCkgJT4lIA0KICBzZXRfdGlkeV9uYW1lcyhzeW50YWN0aWMgPSBUUlVFKSAlPiUgDQogIG11dGF0ZV9pZihpc19sb2dpY2FsLCBmYWN0b3IsIGxldmVscz1jKCJGQUxTRSIsIlRSVUUiKSkgJT4lIA0KICBtdXRhdGVfaWYoaXNfY2hhcmFjdGVyLCBhc19mYWN0b3IpICU+JSANCiAgbXV0YXRlX2lmKH5pcy5mYWN0b3IoLikmbl9kaXN0aW5jdCguKSE9MiwgZmN0X2luZnJlcSkgLT4NCiAgYnJlbnRfZHQNCg0KZG9fZ2xtPC0hZmlsZS5leGlzdHMoIi4uL291dHB1dHMvZ2xtcy5yZHMiKQ0KZG9fZ2xtbmV0czwtIWZpbGUuZXhpc3RzKCIuLi9vdXRwdXRzL2dsbW5ldHMucmRzIikNCmRvX3RyZWVzPC0hZmlsZS5leGlzdHMoIi4uL291dHB1dHMvdHJlZXMucmRzIikNCmRvX2F1dG9tbDwtIWZpbGUuZXhpc3RzKCIuLi9vdXRwdXRzL2F1dG90bWwucmRzIikNCmBgYA0KDQojIFNhbXBsZXMNCmBgYHtyfQ0KbGlicmFyeShtb2RlbHIpDQpsaWJyYXJ5KHJlY2lwZXMpDQpgYGANCg0KIyMgQmFzaWMgc2FtcGxlcw0KDQpQZXJmb3JtIGEgYmFzaWMgNzA6MzAgc3BsaXQNCg0KYGBge3J9DQpzZXQuc2VlZCgyMDE4MDQwOSkNCmJyZW50X2R0ICU+JSANCiAgbW9kZWxyOjpyZXNhbXBsZV9wYXJ0aXRpb24oYyh0cmFpbj0wLjcsdGVzdD0wLjMpKSAtPg0KICBzcGxpdHMNCg0Kc3BsaXRzICU+JSANCiAgcGx1Y2soInRyYWluIikgJT4lIA0KICBhc190aWJibGUoKS0+DQogIHRyYWluX3Jhdw0KDQpzcGxpdHMgJT4lIA0KICBwbHVjaygidGVzdCIpICU+JSANCiAgYXNfdGliYmxlKCktPg0KICB0ZXN0X3Jhdw0KYGBgDQoNClN0YXJ0IG91ciByZWNpcGUgZm9yIHByb2Nlc3Npbmcgb3VyIGRhdGENCg0KYGBge3J9DQp0cmFpbl9yYXcgJT4lIA0KICByZWNpcGUoIC4pICAlPiUNCiAgYWRkX3JvbGUoYW55X3B1cmNoYXNlLCBuZXdfcm9sZSA9ICJvdXRjb21lIikgJT4lIA0KICBhZGRfcm9sZShlbWFpbF91c2VyX2lkLCBuZXdfcm9sZSA9ICJpZCIpICU+JSANCiAgYWRkX3JvbGUoZXZlcnl0aGluZygpLC1hbnlfcHVyY2hhc2UsIC1lbWFpbF91c2VyX2lkLCBuZXdfcm9sZSA9ICJwcmVkaWN0b3IiKS0+DQogIHN0YXJ0ZXJfcmVjaXBlDQoNCnN0YXJ0ZXJfcmVjaXBlDQpgYGANCg0KUGVyZm9ybSBzb21lIHN0ZXBzIHRvIHJlbW92ZSBjb2x1bW5zIGFuZCBmaWxsIGluIG1pc3NpbmcgY291bnRyeQ0KYGBge3J9DQpzdGFydGVyX3JlY2lwZSAlPiUgDQogIHN0ZXBfcm0oaGFzX3JvbGUoImlkIikpICU+JSANCiAgc3RlcF9jb3JyKGFsbF9udW1lcmljKCkpICAlPiUgDQogIHN0ZXBfYmFnaW1wdXRlKGNvdW50cnlfY29kZSwgc2VlZF92YWwgPSA0MikgJT4lIA0KICBzdGVwX290aGVyKGNvdW50cnlfY29kZSkgJT4lICANCiAgc3RlcF96dihhbGxfcHJlZGljdG9ycygpKSAgJT4lIA0KICBwcmVwKHRyYWluX3JhdykgLT4NCiAgZmlsdGVyX3JlY2lwZQ0KDQpmaWx0ZXJfcmVjaXBlDQpgYGANCg0KYGBge3J9DQp0cmFpbl9iIDwtIGJha2UoZmlsdGVyX3JlY2lwZSwgbmV3ZGF0YSA9IHRyYWluX3JhdykgDQp0ZXN0X2IgIDwtIGJha2UoZmlsdGVyX3JlY2lwZSwgbmV3ZGF0YSA9IHRlc3RfcmF3KSANCg0KdHJhaW5fYg0KYGBgDQoNCg0KDQpQZXJmb3JtIG51bWVyaWMgc3RhbmRhcmRpc2F0aW9uIHN0ZXBzLg0KDQpgYGB7cn0NCnRyYWluX2IgJT4lIA0KICByZWNpcGUoYW55X3B1cmNoYXNlfi4pICU+JSANCiAgc3RlcF9jZW50ZXIoYWxsX251bWVyaWMoKSkgJT4lIA0KICBzdGVwX3NjYWxlKGFsbF9udW1lcmljKCkpICU+JSANCiAgcHJlcCh0cmFpbl9iKSAtPg0KICBzdGFuZGFyZGlzZV9yZWNpcGUNCg0Kc3RhbmRhcmRpc2VfcmVjaXBlDQpgYGANCg0KDQpgYGB7cn0NCnRyYWluX3N0ZCA8LSBiYWtlKHN0YW5kYXJkaXNlX3JlY2lwZSwgdHJhaW5fYikgDQp0ZXN0X3N0ZCAgPC0gYmFrZShzdGFuZGFyZGlzZV9yZWNpcGUsIHRlc3RfYikgDQoNCnRyYWluX3N0ZA0KYGBgDQoNCiMjIE92ZXJzYW1wbGUgdHJhaW5pbmcgcHVyY2hhc2VzIA0KYGBge3J9DQpzZXQuc2VlZCgyMDE4MDEwMSkNCmZpbHRlcl9yZWNpcGUgJT4lIA0KICBzdGVwX3Vwc2FtcGxlKGFsbF9vdXRjb21lcygpLCByYXRpbz0gLjI1KSAlPiUgDQogIHByZXAocmV0YWluPVRSVUUpICU+JSANCiAganVpY2UoKSAlPiUgDQogICMgaGFjayBiZWNhdXNlIGp1aWNlIGlzbid0IHJlZHVjaW5nIHRoZSBjb2x1bW4gc2V0DQogIGJha2UoZmlsdGVyX3JlY2lwZSwgLikgLT4NCiAgdHJhaW5fdXANCg0KcHJpbnQocGFzdGUoIk5ldyByb3dzOiIsIG5yb3codHJhaW5fdXApLSBucm93KHRyYWluX2IpKSkNCmBgYA0KDQpOb3cgc3RhbmRhcmRpc2UgaXQNCmBgYHtyfQ0KdHJhaW5fdXBzIDwtIGJha2Uoc3RhbmRhcmRpc2VfcmVjaXBlLCB0cmFpbl91cCkgDQpgYGANCg0KIyMgU3ludGhlc2lzZWQgdHJhaW5pbmcgcHVyY2hhc2VzIA0KYGBge3J9DQpsaWJyYXJ5KHN5bnRocG9wKQ0KYnJlbnRfZHQgJT4lIA0KICBmaWx0ZXIoYW55X3B1cmNoYXNlPT0iVFJVRSIpICU+JSANCiAgYmFrZShmaWx0ZXJfcmVjaXBlLC4pICU+JSANCiAgc3luKGs9MzAwMDApICAtPg0KICBzeW50aF9wdXJjaGFzZXMNCg0Kc3ludGhfcHVyY2hhc2VzICU+JSANCiAgcGx1Y2soInN5biIpICU+JSANCiAgdW5pb24odHJhaW5fYikgJT4lIA0KICBzYW1wbGVfbihucm93KC4pKSAtPg0KICB0cmFpbl9zeW4NCmBgYA0KDQpgYGB7cn0NCnRyYWluX3N5bl9zdGQ8LSBiYWtlKHN0YW5kYXJkaXNlX3JlY2lwZSx0cmFpbl9zeW4pIA0KYGBgDQoNCiMjIFByZXBhcmluZw0KV2UnbGwgd2FudCB0byBidWlsZCBtb2RlbHMgYWdhaW5zdCBvdXIgdHJhaW5pbmcgc2V0cyBzbyBsZXQncyBzYXZlIHNvbWUgY29kZSBieSBtYWtpbmcgYSBiaWcgbGlzdCB3ZSBjYW4gbWFwIG1vZGVscyB0by4NCg0KYGBge3J9DQp0cmFpbl9zZXRzPC1saXN0KA0KICAiQmFzaWMiPXRyYWluX2IsDQogICJTdGFuZGFyZGlzZWQiPXRyYWluX3N0ZCwNCiAgIlVwc2FtcGxlZCI9dHJhaW5fdXAsDQogICJVcHNhbXBsZWQgJiBTdGFuZGFyZGlzZWQiPXRyYWluX3VwcywNCiAgIlN5bnRoZXNpc2VkIj10cmFpbl9zeW4sDQogICJTeW50aGVzaXNlZCAmIFN0YW5kYXJkaXNlZCIgPXRyYWluX3N5bl9zdGQNCikNCmBgYA0KDQpMZXQncyByZW1vdmUgYSBoaWdoIFJBTSBvYmplY3QuDQoNCmBgYHtyfQ0Kcm0oImZpbHRlcl9yZWNpcGUiKQ0KYGBgDQoNCiMgTW9kZWxzIC0gQmFzaWMgZ2xtcw0KYGBge3IgIGV2YWw9ZG9fZ2xtLCBlY2hvPVRSVUV9DQpsaWJyYXJ5KGJyb29tKQ0KbGlicmFyeShnZ3Bsb3QyKQ0KdHJhaW5fc2V0cyAlPiUgDQogIG1hcCh+Z2xtKGFueV9wdXJjaGFzZX4gIG1lbWJlcl9yYXRpbmcrIEZpcnN0LlJlc3BvbmRlci5LaXQrIFVua25vd24ucmVjZCsgTW9uZGF5LkxpbmtzLi4ub3VyLmZhdm9yaXRlLlNRTC4uLnRlY2gubmV3cy5mcm9tLnRoZS53ZWVrKyB3ZWVrZGF5X29wZW5fcHJvcCArIHdlZWtkYXlfY2xpY2tzX3Byb3AsIA0KICAgICAgICAgICBkYXRhPS4sDQogICAgICAgICAgIGZhbWlseT0iYmlub21pYWwiLA0KICAgICAgICAgICB5PUZBTFNFLHg9RkFMU0UsbW9kZWw9RkFMU0UpKSAgLT4NCiAgYmFzaWNfZ2xtDQoNCnNhdmVSRFMoYmFzaWNfZ2xtLCAiLi4vb3V0cHV0cy9nbG1zLnJkcyIpDQpgYGANCg0KYGBge3IgZXZhbD0hZG9fZ2xtLCBlY2hvPUZBTFNFfQ0KYmFzaWNfZ2xtPC1yZWFkX3JkcygiLi4vb3V0cHV0cy9nbG1zLnJkcyIpDQpgYGANCg0KYGBge3IgZmlnLndpZHRoPTEwfQ0KYmFzaWNfZ2xtICU+JSANCiAgbWFwX2RmKHRpZHksIC5pZCA9ICJzZXQiKSAlPiUgDQogIGZpbHRlcih0ZXJtIT0iKEludGVyY2VwdCkiKSAlPiUgDQogIGdncGxvdChhZXMoeD10ZXJtLCB5PWVzdGltYXRlLCBjb2xvdXI9c2V0KSkgKw0KICBnZW9tX3BvaW50KGFscGhhPS41LCBzaXplPTMpICArDQogIGNvb3JkX2ZsaXAoKSArDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKSsNCiAgZ2VvbV9obGluZShhZXMoeWludGVyY2VwdD0wKSwgY29sb3VyPSJkYXJrZ3JleSIsIGxpbmV0eXBlPSJkYXNoZWQiKQ0KYGBgDQoNCkF0dHJpYnV0ZXMgd2l0aCB2YWx1ZXMgY29uc2lzdGVudGx5IGFib3ZlIDAgKGRhc2hlZCBsaW5lKSBpbmNyZWFzZSBsaWtlbGlob29kIHRvIGJ1eSB0cmFpbmluZywgYW5kIHRob3NlIGJlbG93IHJlZHVjZSB0aGUgbGlrZWxpaG9vZC4gVGhlIGZ1cnRoZXIgYXdheSBmcm9tIHRoZSAwLCB0aGUgZ3JlYXRlciB0aGUgaW1wYWN0LiANCg0KYGBge3J9DQpsaWJyYXJ5KG9wdGlSdW0pDQp2YWxzX3RvX2NvbnY8LXNlcSgtMS41LDEsYnk9MC41KQ0KZGF0YV9mcmFtZShDb2VmZmljaWVudD12YWxzX3RvX2NvbnYsDQogICAgICAgICAgIGBPZGRzIFJhdGlvIChwL3AtMSlgPSByb3VuZChsb2dpdC5vZGQodmFsc190b19jb252KSwyKSwNCiAgICAgICAgICAgYFByb2JhYmlsaXR5YCA9IHJvdW5kKGxvZ2l0LnByb2IodmFsc190b19jb252KSwyKSkNCmBgYA0KDQojIE1vZGVscyAtIEJhc2ljIHRyZWUgbW9kZWxzDQpgYGB7ciBldmFsPWRvX3RyZWVzLCBlY2hvPVRSVUV9DQpsaWJyYXJ5KEZGVHJlZXMpDQp0cmFpbl9zZXRzICU+JSANCiAgbWFwKH5kcGx5cjo6c2VsZWN0KG11dGF0ZSguLCBhbnlfcHVyY2hhc2U9YXMubG9naWNhbChhbnlfcHVyY2hhc2UpKSwNCiAgICAgICAgICAgICAgbWVtYmVyX3JhdGluZzp3ZWVrZW5kX29wZW5fcHJvcCwgDQogICAgICAgICAgICAgIGhvbGlkYXlfY2xpY2tzX3Byb3A6d2Vla2VuZF9jbGlja3NfcHJvcCkpICU+JSANCiAgbWFwKH5GRlRyZWVzKGFueV9wdXJjaGFzZX4uLCAuLA0KICAgICAgICAgICAgICAgZ29hbD0iYmFjYyIsZG8uY29tcCA9IEZBTFNFLHByb2dyZXNzID0gRkFMU0UpKSAtPg0KICBiYXNpY190cmVlcw0Kc2F2ZVJEUyhiYXNpY190cmVlcywgIi4uL291dHB1dHMvdHJlZXMucmRzIikNCmBgYA0KDQpgYGB7ciBldmFsPSFkb19nbG0sIGVjaG89RkFMU0V9DQpiYXNpY190cmVlczwtcmVhZF9yZHMoIi4uL291dHB1dHMvdHJlZXMucmRzIikNCmBgYA0KDQpgYGB7cn0NCmJhc2ljX3RyZWVzICU+JSANCiAgbWFwX2RmKH5jYmluZCguJHRyZWUuZGVmaW5pdGlvbnMsLiR0cmVlLnN0YXRzJHRyYWluKSwuaWQgPSAic2V0IikgLT4NCiAgYmFzaWNfdHJlZV9yZXN1bHRzDQoNCmJhc2ljX3RyZWVfcmVzdWx0cw0KYGBgDQoNCg0KDQpUaGUgY2hhcnQgc2hvd3MgZGlmZmVyZW50IHRyZWVzIGJ1aWx0IHVzaW5nIHRoZSBkaWZmZXJlbnQgZGF0YSBzZXRzIGFuZCB0aGUgYmFsYW5jZWQgYWNjdXJhY3kgbWVhc3VyZS4gQmFsYW5jZWQgYWNjdXJhY3kgaXMgYXZlcmFnZSBvZiB0aGUgcHJvcG9ydGlvbiBvZiBwdXJjaGFzZXMgY29ycmVjdGx5IGNsYXNzaWZpZWQgYW5kIHRoZSBwcm9wb3J0aW9uIG9mIG5vbi1wdXJjaGFzZXMgY29ycmVjdGx5IGNsYXNzaWZpZWQuDQpgYGB7ciBmaWcud2lkdGg9MTB9DQpiYXNpY190cmVlX3Jlc3VsdHMgJT4lIA0KICBnZ3Bsb3QoYWVzKHg9Y3VlcywgeT1iYWNjLCBjb2xvdXI9c2V0KSkgKw0KICBnZW9tX3BvaW50KGFscGhhPS41LCBzaXplPTMpICArDQogIGNvb3JkX2ZsaXAoKSArDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKQ0KYGBgDQoNCldlIGNhbiBsb29rIGF0IHRoZSBkaWZmZXJlbmNlIGJldHdlZW4gdGhlIGNvcnJlY3RseSBjbGFzc2lmaWVkIHB1cmNoYXNlcyAocmVkKSBhbmQgdGhlIGNvcnJlY3RseSBjbGFzc2lmaWVkIG5vbi1wdXJjaGFzZXMgKGdyZXkpIGZvciB0aGUgdHJlZXMgd2l0aCB0aGUgaGlnaGVzdCBiYWxhbmNlZCBhY2N1cmFjeS4gVGhlcmUncyB0d28gdmlzaWJsZSBjYXNlczoNCg0KMS4gaGlnaCBhY2N1cmFjeSBmb3IgcHVyY2hhc2VzIGFuZCBsb3cgYWNjdXJhY3kgZm9yIG5vbi1wdXJjaGFzZXMgDQoyLiBPSyBhY2N1cmFjeSBmb3IgcHVyY2hhc2VzIGFuZCBnb29kIGFjY3VyYWN5IGZvciBub24tcHVyY2hhc2VzDQoNClNlbGVjdGlvbiBvbmUgb2YgdGhlc2Ugd291bGQgZGVwZW5kIG9uIHRoZSBjb3N0IG9mIHRoZSBsb3cgYWNjdXJhY3kgZm9yIG5vbi1wdXJjaGFzZXMuDQoNCmBgYHtyIGZpZy53aWR0aD0xMH0NCmJhc2ljX3RyZWVfcmVzdWx0cyAlPiUgDQogIHRvcF9uKDEwLCBiYWNjKSAlPiUgDQogIG11dGF0ZShjdWVzPXN0cmluZ3I6OnN0cl93cmFwKHN0cmluZ3I6OnN0cl9yZXBsYWNlX2FsbChjdWVzLCI7IiwgIiAiKSwxKSkgJT4lIA0KICBkcGx5cjo6c2VsZWN0KGN1ZXMsIHNlbnMsIHNwZWMsIHNldCkgJT4lIA0KICBnZ3Bsb3QoYWVzKHg9c2V0LCB5PXNwZWMsIHltYXg9c2VucywgeW1pbj1zcGVjKSkgKw0KICBnZW9tX3BvaW50cmFuZ2UoYWxwaGE9MC4zLCBzaXplPTIpKw0KICBnZW9tX3BvaW50KGFlcyh5PXNlbnMpLCBjb2xvdXI9InJlZCIsIHNpemU9NikrDQogIGZhY2V0X3dyYXAofmN1ZXMsIG5jb2w9NCkrDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKQ0KYGBgDQoNCg0KIyBNb2RlbHMgLSBnbG1uZXRzDQoNCmdsbW5ldHMgdGFrZSBhbGwgb3VyIHZhcmlhYmxlcyBhbmQgY29uc3RydWN0IGEgbW9kZWwgd2l0aCB0aGVtLiBIb3dldmVyLCBhIG1vZGVsIHdpdGggYHIgbmNvbCh0ZXN0X2IpLTFgIGNvZWZmaWNpZW50cyB3b3VsZCBiZSBwcmV0dHkgY3JhenkuIEluc3RlYWQgYGdsbW5ldGAgZG93bndlaWdodHMgY29lZmZpY2llbnRzIHRvd2FyZHMgMCBhcyBpdCBwZW5hbGlzZXMgY29tcGxleGl0eS4gQXMgc3VjaCwgZXZlbiB0aG91Z2ggb3VyIG1vZGVsIG5vdGlvbmFsbHkgY29udGFpbnMgYWxsIG91ciBjb2x1bW5zIG5vdCBldmVyeXRoaW5nIHdpbGwgY29udHJpYnV0ZS4NCg0KDQpgYGB7ciBldmFsPWRvX2dsbW5ldHN9DQpsaWJyYXJ5KGdsbW5ldCkNCmxpYnJhcnkoZ2xtbmV0VXRpbHMpDQp0cmFpbl9zZXRzICU+JSANCiAgbWFwKH5jdi5nbG1uZXQoYW55X3B1cmNoYXNlfi4sIGRhdGE9LiAsDQogICAgICAgICAgICAgIGZhbWlseSA9ICJiaW5vbWlhbCIsdHlwZS5tZWFzdXJlPSJjbGFzcyIsDQogICAgICAgICAgICAgIHN0YW5kYXJkaXplID0gRkFMU0UpKSAtPg0KICBiYXNpY19nbG1uZXRzDQoNCnNhdmVSRFMoYmFzaWNfZ2xtbmV0cywgIi4uL291dHB1dHMvZ2xtbmV0cy5yZHMiKQ0KYGBgDQoNCmBgYHtyIGV2YWw9IWRvX2dsbW5ldHMsIGVjaG89RkFMU0V9DQpiYXNpY19nbG1uZXRzPC1yZWFkX3JkcygiLi4vb3V0cHV0cy9nbG1uZXRzLnJkcyIpDQpgYGANCg0KYGBge3IgZmlnLmhlaWdodD0xNX0NCmJhc2ljX2dsbW5ldHMgJT4lIA0KICBtYXBfZGYofnJvd25hbWVzX3RvX2NvbHVtbihhcy5kYXRhLmZyYW1lKGFzLm1hdHJpeChjb2VmZmljaWVudHMoLikpKSwiY29sIiksLmlkID0gInNldCIgKSAlPiUgDQogIHJlbmFtZShjb2VmPWAxYCkgLT4gDQogIGJhc2ljX2dsbW5ldF9jb2Vmcw0KDQpiYXNpY19nbG1uZXRfY29lZnMgJT4lIA0KICBmaWx0ZXIoY29lZiE9MC4wMDAwMDBlKzAwKSAlPiUgDQogIGZpbHRlcihjb2whPSIoSW50ZXJjZXB0KSIpICU+JSANCiAgYXJyYW5nZShjb2wpICU+JSANCiAgY291bnQoY29sKSAlPiUgDQogIG11dGF0ZSggcGFuZSA9IHJvd19udW1iZXIoKSAlLyUgMzgpIC0+DQogIHBhbmVzDQogIA0KYmFzaWNfZ2xtbmV0X2NvZWZzICU+JSANCiAgaW5uZXJfam9pbihwYW5lcykgJT4lIA0KICBhcnJhbmdlKGRlc2MoY29sKSkgJT4lIA0KICBtdXRhdGUoY29sPWZjdF9pbm9yZGVyKGNvbCkpICU+JSANCiAgZ2dwbG90KGFlcyh4PWNvbCwgeT1jb2VmLCBjb2xvdXI9c2V0KSkgKw0KICBnZW9tX3BvaW50KCkgKyANCiAgY29vcmRfZmxpcCgpKw0KICBmYWNldF93cmFwKH5wYW5lLHNjYWxlcyA9ICJmcmVlIiwgbmNvbD01KSsNCiAgZ2d0aGVtZXM6OnRoZW1lX2ZpdmV0aGlydHllaWdodCgpDQpgYGANCg0KVGhlIG5vbi1zdGFuZGFyZGlzZWQgdHlwZXMgb2Ygc2FtcGxlcyBkbyBub3Qgd29yayB3ZWxsIHdpdGggZ2xtbmV0IGFuZCB0eXBpY2FsbHkgaXQgd291bGQgaGF2ZSBzY2FsZWQgdGhlbS4gV2UgY2FuIHNlZSB3aGljaCBjb2x1bW5zIChwcmltYXJpbHkgaW4gdGhlIHN0YW5kYXJkaXNlZCBzYW1wbGVzKSBtYWtlIGl0IG91ciBtb2RlbHMgbW9zdCBjb21tb25seS4gVGhlc2Ugd2lsbCBiZSB0aGUgbW9zdCBwcmVkaWN0aXZlIGNvbHVtbnMgb3ZlcmFsbC4NCmBgYHtyIGZpZy5oZWlnaHQ9MTV9DQpwYW5lcyAlPiUgDQogIGFycmFuZ2UoZGVzYyhuKSkgJT4lIA0KICBtdXRhdGUoY29sPWZjdF9pbm9yZGVyKGNvbCkpICU+JSANCiAgbXV0YXRlKCBwYW5lID0gcm93X251bWJlcigpICUvJSAzOCkgJT4lIA0KICBhcnJhbmdlKGRlc2MoY29sKSkgJT4lIA0KICBtdXRhdGUoY29sPWZjdF9pbm9yZGVyKGNvbCkpICU+JSANCiAgZ2dwbG90KGFlcyh4PWNvbCwgeT1uKSkgKw0KICBnZW9tX2NvbCgpICsgDQogIGNvb3JkX2ZsaXAoKSsNCiAgZmFjZXRfd3JhcCh+cGFuZSwgbmNvbD01LCBzY2FsZXMgPSAiZnJlZV95IikrDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKQ0KYGBgDQoNCg0KIyBNb2RlbHMgLSBDb21wbGV4IG1vZGVscyB2aWEgaDJvDQoNCmBgYHtyIGV2YWw9ZG9fYXV0b21sfQ0KbGlicmFyeShoMm8pDQpvcHRpb25zKCJoMm8udXNlLmRhdGEudGFibGUiPVRSVUUpDQpoMm8uaW5pdChtYXhfbWVtX3NpemUgPSAiOEciKQ0KdHJhaW5fc2V0cyAlPiUgDQogIG1hcChhcy5oMm8pICU+JSANCiAgbWFwKH5oMm8uYXV0b21sKHkgPSAiYW55X3B1cmNoYXNlIiwgdHJhaW5pbmdfZnJhbWUgPSAuLCBzZWVkID0gMTMxMywgbWF4X3J1bnRpbWVfc2VjcyA9IDE4MCkpIC0+DQogIGJhc2ljX2F1dG9tbA0KDQpzYXZlUkRTKGJhc2ljX2F1dG9tbCwgIi4uL291dHB1dHMvYXV0b21sLnJkcyIpDQpgYGANCg0KYGBge3IgZXZhbD0hZG9fZ2xtbmV0cywgZWNobz1GQUxTRX0NCmJhc2ljX2F1dG9tbDwtcmVhZF9yZHMoIi4uL291dHB1dHMvYXV0b21sLnJkcyIpDQpgYGANCg0KYGBge3J9DQpiYXNpY19hdXRvbWwgJT4lIA0KICBtYXBfZGYofmFzLmRhdGEuZnJhbWUoLkBsZWFkZXJib2FyZCksIC5pZD0ic2V0IikgIC0+DQogIGF1dG9tbF9oaWdobGV2ZWxfcmVzdWx0cw0KYGBgDQoNCiMgRXZhbHVhdGlvbg0K